Home › Notes › Virtualization › k8s - Kubernetes › Rancher › k3s

k3s

k3s

  • rancher k3s
  • Get postgres on k3s working
    • Is RDS cheap enough
  • https://github.com/k3s-io/k3s
    • Running node args
      # master
k3s.io/node-args=[
"server",
"--flannel-iface","eth0",
"--node-ip","10.20.22.12",
"--node-taint","node-role.kubernetes.io/master=true:NoSchedule",
"--tls-san","10.20.22.222",
"--disable","servicelb",
"--disable","traefik"]

# worker
k3s.io/node-args=[
"agent",
"--server","https://10.20.22.222:6443",
"--token","********",
"--flannel-iface","eth0",
"--node-ip","10.20.22.15"]
    • Troubleshooting
      systemctl cat k3s

cat /etc/systemd/system/k3s.service
cat /etc/systemd/system/k3s.service.d/*.conf  # if any overrides exist

echo 'ip -4 -brief addr' |ssh -t 10.20.22.14 'sudo -i'
ip -4 -brief addr
kubectl -n kube-system get cm kube-vip -o yaml 2>/dev/null \
| grep -i interface

journalctl -u k3s -n 100 --no-pager
top -bn1 | head -20
free -h
df -h

journalctl -u k3s -n 200 --no-pager \
| grep -iE "error|fail|etcd|panic|leader"

journalctl -u k3s -f

cat > /etc/rancher/k3s/config.yaml << EOF
server: https://10.20.22.222/:6443
token: $MYTOKEN
EOF
    • Reading etcd data
      ETCD_VER=v3.5.16
curl -sL https://github.com/etcd-io/etcd/releases/download/${ETCD_VER}/etcd-${ETCD_VER}-linux-amd64.tar.gz \
| tar xz -C /tmp --strip-components=1

/tmp/etcdctl \
--endpoints=https://127.0.0.1:2379 \
--cacert=/var/lib/rancher/k3s/server/tls/etcd/server-ca.crt \
--cert=/var/lib/rancher/k3s/server/tls/etcd/client.crt \
--key=/var/lib/rancher/k3s/server/tls/etcd/client.key \
member list -w table
  • https://github.com/alexellis/k3sup
    # single node on linux 2023
sudo -i
curl -sLS https://get.k3sup.dev | sh
k3sup install --local --context xx \
--ip 192.168.1.202 --k3s-version v1.25.12+k3s1


k3sup install \
--cluster \
--context x \
--k3s-extra-args "--disable traefik --tls-san $LOCAL_IP" \
--k3s-version=v1.24.16+k3s1 \
--local-path ~/.kube/config \
--host x.x.com

export KUBECONFIG=/root/kubeconfig
kubectl config use-context x
kubectl get node -o wide

# uninstall
# /usr/local/bin/k3s-uninstall.sh
  • https://github.com/cnrancher/autok3s
    • run server 
      autok3s serve # on docker2
aws --profile ni-dev sts get-session-token # not needed on latest version
~/.autok3s # configs

docker run \
--name autok3s \
-itd \
--restart=unless-stopped \
-p 8080:8080 \
-v /var/run/docker.sock:/var/run/docker.sock \
cnrancher/autok3s:v0.9.2
    • Exported direct commands
      autok3s create \
--provider aws \
--docker-script https://get.docker.com \
--k3s-channel stable \
--k3s-install-script https://get.k3s.io \
--master 1 \
--name ni-x \
--rollback \
--set [object Object] \
--ssh-key-path /Users/myron/.autok3s/aws/clusters/ni-x.us-east-1.aws/id_rsa \
--ssh-port 22 \
--ssh-user ubuntu \
--worker 0 \
--access-key kkk \
--ami ami-053b0d53c279acc90 --instance-type t3a.medium \
--keypair-name ni-x.us-east-1.aws \
--region us-east-1 \
--root-size 16 \
--secret-key sss \
--security-group sg-0aba133bd21d452d4 \
--subnet-id subnet-ada406f2 \
--volume-type gp3 \
--vpc-id vpc-f774868a \
--zone us-east-1a


autok3s create --provider aws \
--docker-script https://get.docker.com \
--k3s-channel stable \
--k3s-install-script https://get.k3s.io \
--master 1 --name x-stage-k3s --rollback \
--ssh-port 22 --ssh-user ubuntu --worker 0 \
--access-key kkk \
--ami ami-0cbd40f694b804622 \
--instance-type m6i.large \
--region us-west-1 \
--root-size 29 \
--secret-key sss \
--session-token ttt \
--volume-type gp2 \
--zone us-west-1c
  • k3s add ECR access
    export ECR_TOKEN=`aws ecr get-login-password --region "us-west-2"`

# add the ECR repo login
sudo cat<<EOF >> /tmp/registries.yaml
configs:
  262359668809.dkr.ecr.us-west-2.amazonaws.com:
    auth:
        username: AWS
        password: ${ECR_TOKEN}
EOF

sudo mv /tmp/registries.yaml /etc/rancher/k3s/registries.yaml

# force reload k3s
sudo systemctl force-reload k3s

# check if k3s configuration picked up the ECR configuration
sudo cat /var/lib/rancher/k3s/agent/etc/containerd/config.toml

k3s crictl img
k3s ctr i ls

# garbage collect unused images
k3s crictl rmi --prune

aws --region us-west-2 \
ecr list-images \
--repository-name eval-mapper \
--registry-id 262359668809
  • k3s remove Traefik
    sudo rm -rf /var/lib/rancher/k3s/server/manifests/traefik.yaml
helm uninstall traefik traefik-crd -n kube-system
sudo systemctl restart k3s
  • Get updated kubecnf file
    cd /etc/rancher/k3s
cat k3s.yaml
# replace localhost with IP

export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
cat /etc/rancher/k3s/k3s.yaml
  • Check kube config cert expirations
    kubectl config view --raw -o jsonpath='{.users[0].user.client-certificate-data}' | base64 -d | openssl x509 -noout -dates
  • Get server token
    cat /var/lib/rancher/k3s/server/token
# k4
K107292fa2f78ecffa79dx8ba3a98::server:f768WlcJS8R0y1bHoRDAEvxM2u